{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import glob as gb\n",
    "import pandas as pd\n",
    "\n",
    "# from eppy import modeleditor\n",
    "from eppy.modeleditor import IDF\n",
    "import eppy.EPlusInterfaceFunctions.parse_idd as parse_idd\n",
    "\n",
    "iddfile = \"/usr/local/EnergyPlus-9-0-0/Energy+.idd\"\n",
    "IDF.setiddname(iddfile)\n",
    "\n",
    "x = parse_idd.extractidddata(iddfile)\n",
    "\n",
    "useful = x[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some classes are in IDD while not implemented\n",
    "exclude_classes = [\"FaultModel:PressureSensorOffset:OutdoorAir\"]\n",
    "\n",
    "class_names = [k[0][\"idfobj\"] for k in useful]\n",
    "class_names = [x for x in class_names if not x in exclude_classes]\n",
    "\n",
    "# Prepare a dict to hold all mappings\n",
    "# I don't create keys on the fly because that's going to help me determine\n",
    "# the classes I couldn't map\n",
    "subsection_names = {k: None for k in class_names}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We sort it by two rules:\n",
    "# Alphabetically (secondary)\n",
    "# Length (primary): from longest to shortest\n",
    "# We want to avoid starting to replace the \"Output:Meter\" portion of \"Output:Meter:FileOnly\"\n",
    "class_names.sort()\n",
    "class_names.sort(key=len, reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# files = gb.glob('../**/*.tex', recursive=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "io_ref_files = gb.glob(\"../input-output-reference/**/*.tex\", recursive=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read entire I/O"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_lines = {}\n",
    "for p in io_ref_files:\n",
    "    with open(p, \"r\") as f:\n",
    "        lines = f.read().splitlines()\n",
    "        all_lines[p] = lines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find the subsection labels for all objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for class_name in class_names:\n",
    "    for p, lines in all_lines.items():\n",
    "        # Only do the file is the class_name hasn't been found yet\n",
    "        if subsection_names[class_name] is None:\n",
    "            for line in lines:\n",
    "                # Some stuff in in a section rather than subsection\n",
    "                re_subsection = re.compile(r\"\\\\(?:sub)?section{{{c}}}\\\\label{{(.*)}}\".format(c=class_name))\n",
    "                if class_name in line:\n",
    "                    m = re_subsection.search(line)\n",
    "                    if m:\n",
    "                        subsection_names[class_name] = m.groups()[0]\n",
    "                        # No need to keep processing the file, we have what we need\n",
    "                        break\n",
    "                        # print(\"Found subsection\")\n",
    "                    else:\n",
    "                        pass\n",
    "                        # print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Manual fixes\n",
    "subsection_names[\"Shading:Site\"] = \"shadingsite-shadingbuilding\"\n",
    "subsection_names[\"Shading:Building\"] = \"shadingsite-shadingbuilding\"\n",
    "subsection_names[\"Shading:Site:Detailed\"] = \"shadingsitedetailed-shadingbuildingdetailed\"\n",
    "subsection_names[\"Shading:Building:Detailed\"] = \"shadingsitedetailed-shadingbuildingdetailed\"\n",
    "\n",
    "subsection_names[\"Output:Meter\"] = \"outputmeter-and-outputmetermeterfileonly\"\n",
    "subsection_names[\"Output:Meter:MeterFileOnly\"] = \"outputmeter-and-outputmetermeterfileonly\"\n",
    "subsection_names[\"Output:Meter:Cumulative\"] = \"outputmetercumulative-and-outputmetercumulativemeterfileonly\"\n",
    "subsection_names[\"Output:Meter:Cumulative:MeterFileOnly\"] = (\n",
    "    \"outputmetercumulative-and-outputmetercumulativemeterfileonly\"\n",
    ")\n",
    "\n",
    "subsection_names[\"SizingPeriod:WeatherFileDays\"] = \"sizingperiodweatherfiledays\"\n",
    "subsection_names[\"SizingPeriod:WeatherFileConditionType\"] = \"sizingperiodweatherfileconditiontype\"\n",
    "\n",
    "\n",
    "subsection_names[\"SurfaceContaminantSourceAndSink:Generic:PressureDriven\"] = (\n",
    "    \"surfacecontaminantsourceandsinkgenericpressuredriven\"\n",
    ")\n",
    "subsection_names[\"SurfaceContaminantSourceAndSink:Generic:BoundaryLayerDiffusion\"] = (\n",
    "    \"surfacecontaminantsourceandsinkgenericboundarylayerdiffusion\"\n",
    ")\n",
    "subsection_names[\"SurfaceContaminantSourceAndSink:Generic:DepositionVelocitySink\"] = (\n",
    "    \"surfacecontaminantsourceandsinkgenericdepositionvelocitysink\"\n",
    ")\n",
    "subsection_names[\"ZoneContaminantSourceAndSink:Generic:DepositionRateSink\"] = (\n",
    "    \"zonecontaminantsourceandsinkgenericdepositionratesink\"\n",
    ")\n",
    "\n",
    "subsection_names[\"WindowsCalculationEngine\"] = \"windowscalculationengine\"\n",
    "\n",
    "# All of the group objects are detailed in the Auxiliary Program file, but there is a section in I/O\n",
    "# that lists them and says that, so we link to it\n",
    "group_objects = [\n",
    "    \"GroundHeatTransfer:Slab:Materials\",\n",
    "    \"GroundHeatTransfer:Slab:MatlProps\",\n",
    "    \"GroundHeatTransfer:Slab:BoundConds\",\n",
    "    \"GroundHeatTransfer:Slab:BldgProps\",\n",
    "    \"GroundHeatTransfer:Slab:Insulation\",\n",
    "    \"GroundHeatTransfer:Slab:EquivalentSlab\",\n",
    "    \"GroundHeatTransfer:Slab:AutoGrid\",\n",
    "    \"GroundHeatTransfer:Slab:ManualGrid\",\n",
    "    \"GroundHeatTransfer:Slab:XFACE\",\n",
    "    \"GroundHeatTransfer:Slab:YFACE\",\n",
    "    \"GroundHeatTransfer:Slab:ZFACE\",\n",
    "    \"GroundHeatTransfer:Basement:SimParameters\",\n",
    "    \"GroundHeatTransfer:Basement:MatlProps\",\n",
    "    \"GroundHeatTransfer:Basement:Insulation\",\n",
    "    \"GroundHeatTransfer:Basement:SurfaceProps\",\n",
    "    \"GroundHeatTransfer:Basement:BldgData\",\n",
    "    \"GroundHeatTransfer:Basement:Interior\",\n",
    "    \"GroundHeatTransfer:Basement:ComBldg\",\n",
    "    \"GroundHeatTransfer:Basement:EquivSlab\",\n",
    "    \"GroundHeatTransfer:Basement:EquivAutoGrid\",\n",
    "    \"GroundHeatTransfer:Basement:AutoGrid\",\n",
    "    \"GroundHeatTransfer:Basement:ManualGrid\",\n",
    "    \"GroundHeatTransfer:Basement:XFACE\",\n",
    "    \"GroundHeatTransfer:Basement:YFACE\",\n",
    "    \"GroundHeatTransfer:Basement:ZFACE\",\n",
    "]\n",
    "\n",
    "for group_obj in group_objects:\n",
    "    subsection_names[group_obj] = \"group-detailed-ground-heat-transfer\"\n",
    "\n",
    "# Added in #7023 which isn't in develop yet\n",
    "subsection_names[\"Coil:WaterHeating:AirToWaterHeatPump:VariableSpeed\"] = (\n",
    "    \"coil-waterheating-airtowaterheatpump-variablespeed\"\n",
    ")\n",
    "\n",
    "# Also changed in #7023\n",
    "subsection_names[\"Curve:ChillerPartLoadWithLift\"] = \"curvechillerpartloadwithlift\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = pd.Series(subsection_names)\n",
    "print(\"Count of found:\")\n",
    "print(s.notnull().value_counts())\n",
    "\n",
    "print(\"\\nClasses we couldn't map:\")\n",
    "print(s[s.isnull()].index.tolist())"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Write original lines\n",
    "for p, lines in all_lines.items():\n",
    "    with open(p, 'w') as f:\n",
    "        f.write(\"\\n\".join(lines))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Strip trailing spaces and write\n",
    "for p, lines in all_lines.items():\n",
    "    new_lines = []\n",
    "    for line in lines:\n",
    "        new_lines.append(line.rstrip())\n",
    "    with open(p, 'w') as f:\n",
    "        f.write(\"\\n\".join(new_lines))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Some Class Names can be greedy\n",
    "\n",
    "Some are way too greedy: used throughout, and are also english words, eg: 'Zone' which is used 6000 times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all that don't have a \":\" in them (we can assume a ':' makes it quite EnergyPlus-unique)\n",
    "potentially_greedy = sorted([x for x in class_names if \":\" not in x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s_greedy = pd.Series([0] * len(potentially_greedy), index=potentially_greedy)\n",
    "for class_name in s_greedy.index:\n",
    "    for p, lines in all_lines.items():\n",
    "        joined_lines = \"\\n\".join(lines)\n",
    "        count = sum(1 for _ in re.finditer(r\"\\b%s\\b\" % re.escape(class_name), joined_lines))\n",
    "        s_greedy.loc[class_name] += count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s_greedy.sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# These are waaaay too greedy, used throughout, and are also english words\n",
    "# This is the full list of potentially_greedy, where I comment out the ones I want to keep\n",
    "exclude_greedy_class_names = [\n",
    "    #'AirLoopHVAC',\n",
    "    #'AvailabilityManagerAssignmentList',\n",
    "    \"Branch\",\n",
    "    #'BranchList',\n",
    "    \"Building\",\n",
    "    #'CentralHeatPumpSystem',\n",
    "    #'ComfortViewFactorAngles',\n",
    "    #'CondenserEquipmentList',\n",
    "    #'CondenserEquipmentOperationSchemes',\n",
    "    #'CondenserLoop',\n",
    "    #'ConnectorList',\n",
    "    \"Construction\",\n",
    "    #'ConvergenceLimits',\n",
    "    #'CurrencyType',\n",
    "    #'DemandManagerAssignmentList',\n",
    "    #'DistrictCooling',\n",
    "    #'DistrictHeating',\n",
    "    \"Door\",\n",
    "    \"Duct\",\n",
    "    #'ElectricEquipment',\n",
    "    #'EnvironmentalImpactFactors',\n",
    "    #'ExternalInterface',\n",
    "    #'FuelFactors',\n",
    "    #'GasEquipment',\n",
    "    #'GeometryTransform',\n",
    "    #'GlazedDoor',\n",
    "    #'GlobalGeometryRules',\n",
    "    #'HVACSystemRootFindingAlgorithm',\n",
    "    #'HeatBalanceAlgorithm',\n",
    "    #'HotWaterEquipment',\n",
    "    #'InternalMass',\n",
    "    \"Lights\",\n",
    "    \"Material\",\n",
    "    #'NodeList',\n",
    "    #'OtherEquipment',\n",
    "    \"People\",\n",
    "    #'PlantEquipmentList',\n",
    "    #'PlantEquipmentOperationSchemes',\n",
    "    #'PlantLoop',\n",
    "    \"Roof\",\n",
    "    #'RoofIrrigation',\n",
    "    #'RoomAirModelType',\n",
    "    #'RunPeriod',\n",
    "    #'ScheduleTypeLimits',\n",
    "    #'ShadowCalculation',\n",
    "    #'SimulationControl',\n",
    "    #'SteamEquipment',\n",
    "    #'TemperingValve',\n",
    "    \"Timestep\",\n",
    "    \"Version\",\n",
    "    \"Window\",\n",
    "    #'WindowShadingControl',\n",
    "    #'WindowsCalculationEngine',\n",
    "    \"Zone\",\n",
    "    #'ZoneAirContaminantBalance',\n",
    "    #'ZoneAirHeatBalanceAlgorithm',\n",
    "    #'ZoneAirMassFlowConservation',\n",
    "    #'ZoneCrossMixing',\n",
    "    #'ZoneEarthtube',\n",
    "    #'ZoneGroup',\n",
    "    #'ZoneList',\n",
    "    #'ZoneMixing',\n",
    "    #'ZoneRefrigerationDoorMixing',\n",
    "    #'ZoneTerminalUnitList',\n",
    "    #'ZoneThermalChimney'\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Troubleshooting nested problems"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here's the initial problem:\n",
    "\n",
    "    line = 'HVACTemplate:Plant:Chiller:ObjectReference'\n",
    "\n",
    "    class_name = HVACTemplate:Plant:Chiller:ObjectReference\n",
    "\n",
    "    => \\hyperref[hvactemplateplantchillerobjectreference]{HVACTemplate:Plant:Chiller:ObjectReference}\n",
    "\n",
    "    class_name = HVACTemplate:Plant:Chiller\n",
    "\n",
    "    => \\hyperref[hvactemplateplantchillerobjectreference]{\\hyperref[hvactemplateplantchiller]{HVACTemplate:Plant:Chiller}:ObjectReference}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "line = \"\"\"HVACTemplate:Plant:Chiller:ObjectReference should be touched *once*,\n",
    "a \\hyperref[hvactemplateplantchillerobjectreference]{HVACTemplate:Plant:Chiller:ObjectReference} must not be touched\"\"\"\n",
    "print(\"Before:\")\n",
    "print(line)\n",
    "for class_name in [\"HVACTemplate:Plant:Chiller:ObjectReference\", \"HVACTemplate:Plant:Chiller\"]:\n",
    "    # We don't want to add an hyperref if there's already one obviously\n",
    "    # So we use a negative look behind to make sure we don't already have the right link done\n",
    "    # And we use a negative look ahead so not match objects that have the same common denominator\n",
    "    # (eg: don't replace 'Output:Meter' inside 'Output:Meter:FileOnly')\n",
    "    noref_re = re.compile(\n",
    "        r\"(?<!\\\\hyperref\\[{l}\\]{{){c}(?!\\:)\".format(c=re.escape(class_name), l=re.escape(subsection_names[class_name]))\n",
    "    )\n",
    "\n",
    "    noref_repl = r\"\\\\hyperref[{l}]{{{c}}}\".format(c=class_name, l=subsection_names[class_name])\n",
    "\n",
    "    line = noref_re.sub(noref_repl, line)\n",
    "\n",
    "print(\"\\nAfter:\")\n",
    "print(line)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Do!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook as tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "found_lines = []\n",
    "# for class_name in ['Generator:Photovoltaic', 'Fan:VariableVolume', 'Coil:Cooling:Water']:\n",
    "\n",
    "for class_name in tqdm(class_names, desc=\"Replace Class Names\"):\n",
    "\n",
    "    if class_name in exclude_greedy_class_names:\n",
    "        print(\"Skipping {}\".format(class_name))\n",
    "        continue\n",
    "    re_subsection = re.compile(r\"(\\\\(?:sub){{0,2}}section){{{c}}}\\\\label{{(.*)}}\".format(c=class_name))\n",
    "\n",
    "    # Special case for lstinline, it must end up INSIDE the hyperref\n",
    "    lstinline_re = re.compile(r\"(\\\\lstinline!{}!)\".format(re.escape(class_name)))\n",
    "    lstinline_repl = r\"\\\\hyperref[{}]{{\\1}}\".format(re.escape(subsection_names[class_name]))\n",
    "\n",
    "    # We don't want to add an hyperref if there's already one obviously\n",
    "    # So we use a negative look behind to make sure we don't already have the right link done\n",
    "    # And we use a negative look ahead so not match objects that have the same common denominator\n",
    "    # (eg: don't replace 'Output:Meter' inside 'Output:Meter:FileOnly')\n",
    "    noref_re = re.compile(\n",
    "        r\"(?<!\\\\hyperref\\[{l}]{{){c}(?!\\:)\".format(c=re.escape(class_name), l=re.escape(subsection_names[class_name]))\n",
    "    )\n",
    "\n",
    "    noref_repl = r\"\\\\hyperref[{l}]{{{c}}}\".format(c=class_name, l=subsection_names[class_name])\n",
    "\n",
    "    outputs_re = re.compile(r\"(\\\\(?:sub){1,2}section){Outputs?}\\\\label{.*}\")\n",
    "\n",
    "    # for p, lines in all_lines.items():\n",
    "\n",
    "    for p in io_ref_files:\n",
    "        with open(p, \"r\") as f:\n",
    "            lines = f.read().splitlines()\n",
    "            all_lines[p] = lines\n",
    "\n",
    "        # print(\"\\n\\n\")\n",
    "        # print(p)\n",
    "        # We skip everything that inside these blocks: lstlisting, equation, tables\n",
    "        inside_lstlisting = False\n",
    "        inside_eq = False\n",
    "        inside_table = False\n",
    "\n",
    "        # We also don't want to add cross references to an object within its own section...\n",
    "        inside_section = False\n",
    "        # We need to capture the actual type (eg: section, subsection) to find\n",
    "        # the next one so we know we're out\n",
    "        section_type = None\n",
    "        # One time flag, so we don't use a regex on all lines to find the section\n",
    "        section_found = False\n",
    "\n",
    "        # And we don't want to do it in the \"Outputs\" section\n",
    "        inside_outputs = False\n",
    "        # Based on the type of section the \"Outputs\" was found\n",
    "        # We search for the next higher level\n",
    "        next_closing_section = False\n",
    "\n",
    "        new_lines = []\n",
    "\n",
    "        for line in lines:\n",
    "\n",
    "            if not section_found:\n",
    "                m = re_subsection.search(line)\n",
    "                if m:\n",
    "                    # print(\"Found the section in {}\".format(p))\n",
    "                    section_found = True\n",
    "                    inside_section = True\n",
    "                    section_type = m.groups()[0]\n",
    "\n",
    "            elif inside_section:\n",
    "                if section_type in line:\n",
    "                    inside_section = False\n",
    "\n",
    "            if r\"\\begin{lstlisting}\" in line:\n",
    "                inside_lstlisting = True\n",
    "            if r\"\\end{lstlisting}\" in line:\n",
    "                inside_lstlisting = False\n",
    "\n",
    "            if r\"\\begin{equation}\" in line:\n",
    "                inside_eq = True\n",
    "            if r\"\\end{equation}\" in line:\n",
    "                inside_eq = False\n",
    "\n",
    "            if (r\"\\begin{table}\" in line) or (r\"\\begin{longtable}\" in line):\n",
    "                inside_table = True\n",
    "            if (r\"\\end{table}\" in line) or (r\"\\end{longtable}\" in line):\n",
    "                inside_table = False\n",
    "\n",
    "            m = outputs_re.search(line)\n",
    "            if m:\n",
    "                inside_outputs = True\n",
    "                output_section_type = m.groups()[0]\n",
    "                next_closing_section_type = output_section_type.replace(\"sub\", \"\", 1)\n",
    "            elif inside_outputs:\n",
    "                if next_closing_section_type in line:\n",
    "                    inside_outputs = False\n",
    "\n",
    "            # If we're not inside a section/equation/table/lstlisting/Outputs\n",
    "            # And class name is inside the line\n",
    "            # And there is no \\label (workaround to avoid paragraphs too)\n",
    "            if (class_name in line) and not (\n",
    "                inside_lstlisting | inside_eq | inside_table | inside_section | inside_outputs | (\"\\label\" in line)\n",
    "            ):\n",
    "                found_lines.append(line)\n",
    "                # Start with the listinline one, lstinline has to be inside the hyperref text portion\n",
    "                new_line = lstinline_re.sub(lstinline_repl, line)\n",
    "                # textbf and emph are around, just like classic,\n",
    "                # so we can do it for all others now\n",
    "                new_line = noref_re.sub(noref_repl, line)\n",
    "                new_lines.append(new_line)\n",
    "\n",
    "            else:\n",
    "                # Do nothing\n",
    "                new_lines.append(line)\n",
    "\n",
    "        # Write new file\n",
    "        with open(p, \"w\") as f:\n",
    "            f.write(\"\\n\".join(new_lines))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
